{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## [Chapter 15: Foundation Models and Emerging Search Paradigms]\n",
    "## Generative AI and the Search Frontier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/thinc/compat.py:36: UserWarning: 'has_mps' is deprecated, please use 'torch.backends.mps.is_built()'\n",
      "  hasattr(torch, \"has_mps\")\n",
      "/opt/conda/lib/python3.10/site-packages/thinc/compat.py:37: UserWarning: 'has_mps' is deprecated, please use 'torch.backends.mps.is_built()'\n",
      "  and torch.has_mps  # type: ignore[attr-defined]\n"
     ]
    }
   ],
   "source": [
    "import html\n",
    "import json\n",
    "import pickle\n",
    "import sys\n",
    "import warnings\n",
    "sys.path.append(\"..\")\n",
    "\n",
    "import pandas\n",
    "import spacy\n",
    "from aips import *\n",
    "from aips.spark import create_view_from_collection\n",
    "from IPython.display import HTML, display\n",
    "from pyspark.sql import SparkSession\n",
    "from pyspark.sql.functions import UserDefinedFunction\n",
    "from pyspark.sql.types import ArrayType, FloatType\n",
    "import numpy\n",
    "\n",
    "warnings.filterwarnings(\"ignore\") #Some operations warn inside a loop, we'll only need to see the first warning\n",
    "\n",
    "engine = get_engine()\n",
    "outdoors_collection = engine.get_collection(\"outdoors\")\n",
    "spark = SparkSession.builder.appName(\"AIPS\").getOrCreate()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Note**: This notebook depends upon the Outdoors dataset. If you have any issues, please rerun the [Setting up the Outdoors Dataset](../ch13/1.setting-up-the-outdoors-dataset.ipynb) notebook."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Mocked Generative Output\n",
    "\n",
    "For this notebook, we mock responses from a large generative model.\n",
    "\n",
    "We encourage you to explore connecting to a generative model API of your choice, and comparing results!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import html\n",
    "mockedGenerativeResponses = pandas.read_csv(\"chapters/ch15/mockedGenerativeResponses.csv\")\n",
    "def get_generative_response(prompt):\n",
    "    prompt=prompt.replace(\"\\n\", \"\\n\") #marshall new lines into consistent format\n",
    "    response = mockedGenerativeResponses.loc[mockedGenerativeResponses[\"prompt\"] == prompt, \"response\"].values\n",
    "    if len(response) > 0:\n",
    "        output = f'''\n",
    "        <div style=\"overflow: auto; margin-bottom: 10px;\">\n",
    "          <h3 style=\"float: left; width: 100px; margin-right: 10px;\">Query:</h3>\n",
    "          <p style=\"overflow: hidden; margin-left: 110px;\"><pre>{html.escape(prompt)}<pre></p>\n",
    "        </div>\n",
    "\n",
    "        <div style=\"overflow: auto; margin-bottom: 10px;\">\n",
    "          <h3 style=\"float: left; width: 100px; margin-right: 10px;\">Response:</h3>\n",
    "          <p style=\"overflow: hidden; margin-left: 110px;\"><pre>{html.escape(response[0])}</pre></p>\n",
    "        </div>\n",
    "        '''\n",
    "        display(HTML(output))\n",
    "        return response[0]\n",
    "    else:\n",
    "        print(f\"\\n\\nSorry! your prompt does not have a mocked value.\")\n",
    "        return \"\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Retriever from Listing 14.6\n",
    "\n",
    "See [../ch14/2.ch14-question-answering-CPU-data-preparation.ipynb#Listing-14.6](../ch14/2.ch14-question-answering-CPU-data-preparation.ipynb#Listing-14.6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp = spacy.load(\"en_core_web_sm\")\n",
    "nlp.remove_pipe(\"ner\")\n",
    "nlp.add_pipe(\"merge_noun_chunks\")\n",
    "determiners = \"all an another any both del each either every half la many much nary neither no some such that the them these this those\".split(\" \")\n",
    "def get_query_from_question(question):\n",
    "    query = []\n",
    "    doc = nlp(question)\n",
    "    for tok in doc:\n",
    "        if tok.pos_ in [\"NOUN\",\"VERB\"]:\n",
    "            query.append(tok.text)\n",
    "    if not len(query):\n",
    "        query = [question]\n",
    "    query = \" \".join(query)\n",
    "    for d in determiners:\n",
    "        query = query.replace(\" \"+d+\" \",\"\")\n",
    "    return query"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def retriever(question):\n",
    "    contexts={\"id\":[],\"question\":[],\"context\":[],\"url\":[]}\n",
    "    query = get_query_from_question(question)\n",
    "    request = {\n",
    "        \"query\": query,\n",
    "        \"query_fields\": [\"body\"],\n",
    "        \"return_fields\": [\"id\", \"url\", \"body\"],\n",
    "        \"filters\": [(\"post_type\", \"answer\")],\n",
    "        \"limit\": 5\n",
    "    }\n",
    "    docs = outdoors_collection.search(**request)[\"docs\"]\n",
    "    for doc in docs:\n",
    "        contexts[\"id\"].append(doc[\"id\"])\n",
    "        contexts[\"url\"].append(doc[\"url\"])\n",
    "        contexts[\"question\"].append(question)\n",
    "        contexts[\"context\"].append(doc[\"body\"])\n",
    "    return pandas.DataFrame(contexts)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Listing 15.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <div style=\"overflow: auto; margin-bottom: 10px;\">\n",
       "          <h3 style=\"float: left; width: 100px; margin-right: 10px;\">Query:</h3>\n",
       "          <p style=\"overflow: hidden; margin-left: 110px;\"><pre>What is a unicorn?<pre></p>\n",
       "        </div>\n",
       "\n",
       "        <div style=\"overflow: auto; margin-bottom: 10px;\">\n",
       "          <h3 style=\"float: left; width: 100px; margin-right: 10px;\">Response:</h3>\n",
       "          <p style=\"overflow: hidden; margin-left: 110px;\"><pre>A unicorn is a mythical creature that is typically depicted as a horse \n",
       "with a single, spiraled horn projecting from its forehead. Unicorns have \n",
       "been a part of human mythology and folklore for centuries and are often \n",
       "associated with purity, grace, and magic. They are said to possess healing \n",
       "powers and to be able to bring good luck to those who encounter them. \n",
       "While unicorns are not real animals, they continue to be a popular symbol \n",
       "in modern culture, appearing in literature, art, and popular media.</pre></p>\n",
       "        </div>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "r = get_generative_response(\"What is a unicorn?\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Listing 15.2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <div style=\"overflow: auto; margin-bottom: 10px;\">\n",
       "          <h3 style=\"float: left; width: 100px; margin-right: 10px;\">Query:</h3>\n",
       "          <p style=\"overflow: hidden; margin-left: 110px;\"><pre>You are a chatbot named AIPS chat. \n",
       "You have a Ph.D. in biology. \n",
       "What is a unicorn?<pre></p>\n",
       "        </div>\n",
       "\n",
       "        <div style=\"overflow: auto; margin-bottom: 10px;\">\n",
       "          <h3 style=\"float: left; width: 100px; margin-right: 10px;\">Response:</h3>\n",
       "          <p style=\"overflow: hidden; margin-left: 110px;\"><pre>Hello! I am Chatbot named AIPS Chat, and I have a Ph.D. in biology. While \n",
       "unicorns have been depicted in various cultures throughout history, they \n",
       "are not real animals that exist in the natural world.\n",
       "\n",
       "From a biological perspective, a unicorn can be described as a mythical \n",
       "creature resembling a horse with a single, spiraled horn on its forehead. \n",
       "However, since unicorns are purely fictional, there is no actual \n",
       "scientific data or research that can be conducted on them.\n",
       "\n",
       "It is important to note that while many animals possess unique \n",
       "characteristics, such as the horn of a rhinoceros or the tusk of a \n",
       "narwhal, these features are not magical and do not provide any special \n",
       "abilities or powers to the animal possessing them.\n",
       "\n",
       "In conclusion, while unicorns are a fascinating aspect of human \n",
       "folklore and mythology, they do not have a basis in reality within the \n",
       "field of biology.</pre></p>\n",
       "        </div>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "r = get_generative_response(\"\"\"You are a chatbot named AIPS chat. \n",
    "You have a Ph.D. in biology. \n",
    "What is a unicorn?\"\"\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Listing 15.3 & Listing 15.4\n",
    "(The prompt and responses for Listings 15.3 and 15.4 are both represented in one cell)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <div style=\"overflow: auto; margin-bottom: 10px;\">\n",
       "          <h3 style=\"float: left; width: 100px; margin-right: 10px;\">Query:</h3>\n",
       "          <p style=\"overflow: hidden; margin-left: 110px;\"><pre>Web search results:\n",
       "\n",
       "[1] &quot;A large language model, or LLM, is a deep learning algorithm that \n",
       "can recognize, summarize, translate, predict and generate text and other \n",
       "content based on knowledge gained from massive datasets.&quot;\n",
       "URL: https://blogs.nvidia.com/blog/2023/01/26/what-are-large-language-models-used-for/\n",
       "\n",
       "[2] A large language model (LLM) is a language model consisting of a \n",
       "neural network with many parameters (typically billions of weights or \n",
       "more), trained on large quantities of unlabeled text using \n",
       "self-supervised learning. LLMs emerged around 2018 and perform well at a \n",
       "wide variety of tasks. This has shifted the focus of natural language \n",
       "processing research away from the previous paradigm of training \n",
       "specialized supervised models for specific tasks.\n",
       "URL: https://en.wikipedia.org/wiki/Large_language_model\n",
       "\n",
       "[3] &quot;In this guide, we&#x27;ll discuss everything you need to know about \n",
       "Large Language Models (LLMs), including key terms, algorithms, \n",
       "fine-tuning, and more. As ChatGPT has taken the internet by storm crossing \n",
       "1 million users in its first 5 days, you may be wondering what machine \n",
       "learning algorithm is running under the hood. While ChatGPT uses a \n",
       "specific type of reinforcement learning called &quot;Reinforcement Learning \n",
       "from Human Feedback (RLHF)&quot;, at a high level it is an example of a \n",
       "Large Language Model (LLM).&quot;\n",
       "URL: https://www.mlq.ai/what-is-a-large-language-model-llm/\n",
       "\n",
       "Instructions: Using the provided web search results, write a comprehensive \n",
       "reply to the given query. Make sure to cite results using \n",
       "[[number](URL)] notation after the reference. If the provided search \n",
       "results refer to multiple subjects with the same name, write separate \n",
       "answers for each subject.\n",
       "\n",
       "Query: What is a large language model?<pre></p>\n",
       "        </div>\n",
       "\n",
       "        <div style=\"overflow: auto; margin-bottom: 10px;\">\n",
       "          <h3 style=\"float: left; width: 100px; margin-right: 10px;\">Response:</h3>\n",
       "          <p style=\"overflow: hidden; margin-left: 110px;\"><pre>A large language model (LLM) is a deep learning algorithm that can \n",
       "recognize, summarize, translate, predict, and generate text and other \n",
       "content based on knowledge gained from massive datasets [[1](https://blogs.nvidia.com/blog/2023/01/26/what-are-large-language-models-used-for/)]. \n",
       "It consists of a neural network with many parameters (typically billions \n",
       "of weights or more) and is trained on large quantities of unlabeled text \n",
       "using self-supervised learning \n",
       "[[2](https://en.wikipedia.org/wiki/Large_language_model)]. LLMs emerged \n",
       "around 2018 and have performed well at a wide variety of tasks, shifting \n",
       "the focus of natural language processing research away from the previous \n",
       "paradigm of training specialized supervised models for specific tasks \n",
       "[[2](https://en.wikipedia.org/wiki/Large_language_model)]. One example of \n",
       "a LLM is ChatGPT, which uses a specific type of reinforcement learning \n",
       "called &quot;Reinforcement Learning from Human Feedback (RLHF)&quot; \n",
       "[[3](https://www.mlq.ai/what-is-a-large-language-model-llm/)].</pre></p>\n",
       "        </div>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "r = get_generative_response(\"\"\"Web search results:\n",
    "\n",
    "[1] \"A large language model, or LLM, is a deep learning algorithm that \n",
    "can recognize, summarize, translate, predict and generate text and other \n",
    "content based on knowledge gained from massive datasets.\"\n",
    "URL: https://blogs.nvidia.com/blog/2023/01/26/what-are-large-language-models-used-for/\n",
    "\n",
    "[2] A large language model (LLM) is a language model consisting of a \n",
    "neural network with many parameters (typically billions of weights or \n",
    "more), trained on large quantities of unlabeled text using \n",
    "self-supervised learning. LLMs emerged around 2018 and perform well at a \n",
    "wide variety of tasks. This has shifted the focus of natural language \n",
    "processing research away from the previous paradigm of training \n",
    "specialized supervised models for specific tasks.\n",
    "URL: https://en.wikipedia.org/wiki/Large_language_model\n",
    "\n",
    "[3] \"In this guide, we'll discuss everything you need to know about \n",
    "Large Language Models (LLMs), including key terms, algorithms, \n",
    "fine-tuning, and more. As ChatGPT has taken the internet by storm crossing \n",
    "1 million users in its first 5 days, you may be wondering what machine \n",
    "learning algorithm is running under the hood. While ChatGPT uses a \n",
    "specific type of reinforcement learning called \"Reinforcement Learning \n",
    "from Human Feedback (RLHF)\", at a high level it is an example of a \n",
    "Large Language Model (LLM).\"\n",
    "URL: https://www.mlq.ai/what-is-a-large-language-model-llm/\n",
    "\n",
    "Instructions: Using the provided web search results, write a comprehensive \n",
    "reply to the given query. Make sure to cite results using \n",
    "[[number](URL)] notation after the reference. If the provided search \n",
    "results refer to multiple subjects with the same name, write separate \n",
    "answers for each subject.\n",
    "\n",
    "Query: What is a large language model?\"\"\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Listing 15.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <div style=\"overflow: auto; margin-bottom: 10px;\">\n",
       "          <h3 style=\"float: left; width: 100px; margin-right: 10px;\">Query:</h3>\n",
       "          <p style=\"overflow: hidden; margin-left: 110px;\"><pre>Web search results:\n",
       "\n",
       "[1] &quot;A large language model, or LLM, is a deep learning algorithm that \n",
       "can recognize, summarize, translate, predict and generate text and other \n",
       "content based on knowledge gained from massive datasets.&quot;\n",
       "URL: https://blogs.nvidia.com/blog/2023/01/26/what-are-large-language-models-used-for/\n",
       "\n",
       "[2] A large language model (LLM) is a language model consisting of a \n",
       "neural network with many parameters (typically billions of weights or \n",
       "more), trained on large quantities of unlabeled text using \n",
       "self-supervised learning. LLMs emerged around 2018 and perform well at a \n",
       "wide variety of tasks. This has shifted the focus of natural language \n",
       "processing research away from the previous paradigm of training \n",
       "specialized supervised models for specific tasks.\n",
       "URL: https://en.wikipedia.org/wiki/Large_language_model\n",
       "\n",
       "[3] &quot;In this guide, we&#x27;ll discuss everything you need to know about \n",
       "Large Language Models (LLMs), including key terms, algorithms, \n",
       "fine-tuning, and more. As ChatGPT has taken the internet by storm crossing \n",
       "1 million users in its first 5 days, you may be wondering what machine \n",
       "learning algorithm is running under the hood. While ChatGPT uses a \n",
       "specific type of reinforcement learning called &quot;Reinforcement Learning \n",
       "from Human Feedback (RLHF)&quot;, at a high level it is an example of a \n",
       "Large Language Model (LLM).&quot;\n",
       "URL: https://www.mlq.ai/what-is-a-large-language-model-llm/\n",
       "\n",
       "Instructions: Using the provided web search results, write a comprehensive \n",
       "reply to the given query. Make sure to cite results using \n",
       "[[number](URL)] notation after the reference. If the provided search \n",
       "results refer to multiple subjects with the same name, write separate \n",
       "answers for each subject.\n",
       "\n",
       "Query: What is a large language model?  Be concise.<pre></p>\n",
       "        </div>\n",
       "\n",
       "        <div style=\"overflow: auto; margin-bottom: 10px;\">\n",
       "          <h3 style=\"float: left; width: 100px; margin-right: 10px;\">Response:</h3>\n",
       "          <p style=\"overflow: hidden; margin-left: 110px;\"><pre>A large language model (LLM) is a deep learning algorithm that uses neural networks with billions of parameters, trained on massive unlabeled text datasets for various language tasks [[2](https://en.wikipedia.org/wiki/Large_language_model)][[1](https://blogs.nvidia.com/blog/2023/01/26/what-are-large-language-models-used-for/)].</pre></p>\n",
       "        </div>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "r = get_generative_response(\"\"\"Web search results:\n",
    "\n",
    "[1] \"A large language model, or LLM, is a deep learning algorithm that \n",
    "can recognize, summarize, translate, predict and generate text and other \n",
    "content based on knowledge gained from massive datasets.\"\n",
    "URL: https://blogs.nvidia.com/blog/2023/01/26/what-are-large-language-models-used-for/\n",
    "\n",
    "[2] A large language model (LLM) is a language model consisting of a \n",
    "neural network with many parameters (typically billions of weights or \n",
    "more), trained on large quantities of unlabeled text using \n",
    "self-supervised learning. LLMs emerged around 2018 and perform well at a \n",
    "wide variety of tasks. This has shifted the focus of natural language \n",
    "processing research away from the previous paradigm of training \n",
    "specialized supervised models for specific tasks.\n",
    "URL: https://en.wikipedia.org/wiki/Large_language_model\n",
    "\n",
    "[3] \"In this guide, we'll discuss everything you need to know about \n",
    "Large Language Models (LLMs), including key terms, algorithms, \n",
    "fine-tuning, and more. As ChatGPT has taken the internet by storm crossing \n",
    "1 million users in its first 5 days, you may be wondering what machine \n",
    "learning algorithm is running under the hood. While ChatGPT uses a \n",
    "specific type of reinforcement learning called \"Reinforcement Learning \n",
    "from Human Feedback (RLHF)\", at a high level it is an example of a \n",
    "Large Language Model (LLM).\"\n",
    "URL: https://www.mlq.ai/what-is-a-large-language-model-llm/\n",
    "\n",
    "Instructions: Using the provided web search results, write a comprehensive \n",
    "reply to the given query. Make sure to cite results using \n",
    "[[number](URL)] notation after the reference. If the provided search \n",
    "results refer to multiple subjects with the same name, write separate \n",
    "answers for each subject.\n",
    "\n",
    "Query: What is a large language model?  Be concise.\"\"\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Listing 15.6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "What are some queries that should find the following documents?  List at least 5 unique queries, where these documents are better than others in an outdoors question and answer dataset.  Be concise and only output the list of queries, and a result number in the format [n] for the best result in the resultset.  Don't print a relevance summary at the end.\n",
      "\n",
      "### Results:\n",
      "{resultset}\n"
     ]
    }
   ],
   "source": [
    "summarize_search_prompt = \"\"\"What are some queries that should find the following documents?  List at least 5 unique queries, where these documents are better than others in an outdoors question and answer dataset.  Be concise and only output the list of queries, and a result number in the format [n] for the best result in the resultset.  Don't print a relevance summary at the end.\n",
    "\n",
    "### Results:\n",
    "{resultset}\"\"\"\n",
    "print(summarize_search_prompt)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Listing 15.7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0. Minimalist shoes or \"barefoot\" shoes are shoes that provide your feet with some form of protection, but get you as close to a barefoot experience as possible. The styles range from simple ultralight sneakers, to thin almost slipper like shoes, all the way down to stick on shoes: image source image source\n",
      "1. There was actually a project done on the definition of what a minimalist shoe is and the result was \"Footwear providing minimal interference with the natural movement of the foot due to its high flexibility, low heel to toe drop, weight and stack height, and the absence of motion control and stability devices\". If you are looking for a simpler definition, this is what Wikipedia says, Minimalist shoes are shoes intended to closely approximate barefoot running conditions. 1 They have reduced cushioning, thin soles, and are of lighter weight than other running shoes, allowing for more sensory contact for the foot on the ground while simultaneously providing the feet with some protection from ground hazards and conditions (such as pebbles and dirt). One example of minimalistic shoes would be the Vibram FiveFingers shoes which look like this. Image Source\n",
      "2. One summer job, I needed shoes to walk on a rocky beach, sometimes in the water, for 5 to 10 miles per day all summer. Stretchy neoprene shoes were terrible for this- no traction and no support. So I used regular sneakers. I chose a pair of cross country racing flats. The model I used are no longer made, but were made by New Balance. The uppers were extremely well ventilated polyester, so they drained very quickly, and the thin material dried much faster than a padded sandal, and certainly much faster than a regular sneaker of leather or cotton would. There are probably other minimalist sneakers that would be equally good. The thing to look for is thin fabric that attaches directly to the sole, with no rubber rim that would keep the water from draining. Running shoes in general will be good because those tend to be very well ventilated, and minimalist shoes will be best because there will be very little padding to hold extra water.\n",
      "3. The answer to this question will vary on your level of foot strength and the terrain. Human beings did pretty well for thousands of years without supportive footwear, but when they started walking on hard stone and climbing rocky mountains they quickly discovered their feet weren't suited for it. You'll notice that most large mammals that live at high elevation have hooves. Most people entering the minimalist barefoot shoe niche grew up wearing sneakers that provided some level of support for your arch/ankle/etc., which has the byproduct of under-developing the stabilizer muscles in your feet as you grow and mature. The result is very sore feet after using the minimalist shoes for long periods of time; the risk is causing a muskuloskeletal disorder from straining these muscles by using them excessively before they have the opportunity to properly develop. It's not unhealthy to wear minimalist footwear, but on what terrain your wear them could be bad for your body in the long run. Human beings were never meant to walk or run exclusively on hard pavement or sidewalks. Nor were we designed to clamber around on sharp rocks at high elevations. Homo Sapiens are a low-elevation tropical animals, You'll notice the vast majority of the world's population is still in Southern Asia, which is very tropical. We had to adapt technology like clothes and footwear in order to thrive in other environments. If you're running on soft ground and you have the foot strength, then there are plenty of arguments in favour of minimalist shoes being better for you than other shoes, because it brings your posture and gait back to what nature intended it to be. If you're hiking in the mountains on uneven rocky terrain, especially while carrying a heavy bag, then you'd be better off wearing a supportive hiking boot, emulating a sturdy hoof of a high-altitude ungulate.\n",
      "4. Barefoot Shoes Also known as minimalist shoes, the theory behind them is to give you as close to a barefoot feel without actually putting the soft, soft flesh of your terribly urbanized and unadapted soles of your feet at risk of damage. My favourite barefoot shoes are Vibram Five-Fingers , I wear either my Mouri's or my KSO's at the beach. Vibram Five-Fingers 'Signa' watersport shoes: The thin sole will be enough to protect your feet from the lava sand, and it will still feel like running on the beach barefoot, with the exception of your feet getting exfoliated by the sand. They also have running specific five fingers if you're interested.\n"
     ]
    }
   ],
   "source": [
    "example_contexts = retriever(\"What are minimalist shoes?\")\n",
    "resultset = [f\"{idx}. {ctx}\" for idx, ctx\n",
    "             in enumerate(list(example_contexts[0:5][\"context\"]))]\n",
    "print(\"\\n\".join(resultset))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Listing 15.8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <div style=\"overflow: auto; margin-bottom: 10px;\">\n",
       "          <h3 style=\"float: left; width: 100px; margin-right: 10px;\">Query:</h3>\n",
       "          <p style=\"overflow: hidden; margin-left: 110px;\"><pre>What are some queries that should find the following documents?  List at least 5 unique queries, where these documents are better than others in an outdoors question and answer dataset.  Be concise and only output the list of queries, and a result number in the format [n] for the best result in the resultset.  Don&#x27;t print a relevance summary at the end.\n",
       "\n",
       "### Results:\n",
       "0. Minimalist shoes or &quot;barefoot&quot; shoes are shoes that provide your feet with some form of protection, but get you as close to a barefoot experience as possible. The styles range from simple ultralight sneakers, to thin almost slipper like shoes, all the way down to stick on shoes: image source image source\n",
       "1. There was actually a project done on the definition of what a minimalist shoe is and the result was &quot;Footwear providing minimal interference with the natural movement of the foot due to its high flexibility, low heel to toe drop, weight and stack height, and the absence of motion control and stability devices&quot;. If you are looking for a simpler definition, this is what Wikipedia says, Minimalist shoes are shoes intended to closely approximate barefoot running conditions. 1 They have reduced cushioning, thin soles, and are of lighter weight than other running shoes, allowing for more sensory contact for the foot on the ground while simultaneously providing the feet with some protection from ground hazards and conditions (such as pebbles and dirt). One example of minimalistic shoes would be the Vibram FiveFingers shoes which look like this. Image Source\n",
       "2. One summer job, I needed shoes to walk on a rocky beach, sometimes in the water, for 5 to 10 miles per day all summer. Stretchy neoprene shoes were terrible for this- no traction and no support. So I used regular sneakers. I chose a pair of cross country racing flats. The model I used are no longer made, but were made by New Balance. The uppers were extremely well ventilated polyester, so they drained very quickly, and the thin material dried much faster than a padded sandal, and certainly much faster than a regular sneaker of leather or cotton would. There are probably other minimalist sneakers that would be equally good. The thing to look for is thin fabric that attaches directly to the sole, with no rubber rim that would keep the water from draining. Running shoes in general will be good because those tend to be very well ventilated, and minimalist shoes will be best because there will be very little padding to hold extra water.\n",
       "3. The answer to this question will vary on your level of foot strength and the terrain. Human beings did pretty well for thousands of years without supportive footwear, but when they started walking on hard stone and climbing rocky mountains they quickly discovered their feet weren&#x27;t suited for it. You&#x27;ll notice that most large mammals that live at high elevation have hooves. Most people entering the minimalist barefoot shoe niche grew up wearing sneakers that provided some level of support for your arch/ankle/etc., which has the byproduct of under-developing the stabilizer muscles in your feet as you grow and mature. The result is very sore feet after using the minimalist shoes for long periods of time; the risk is causing a muskuloskeletal disorder from straining these muscles by using them excessively before they have the opportunity to properly develop. It&#x27;s not unhealthy to wear minimalist footwear, but on what terrain your wear them could be bad for your body in the long run. Human beings were never meant to walk or run exclusively on hard pavement or sidewalks. Nor were we designed to clamber around on sharp rocks at high elevations. Homo Sapiens are a low-elevation tropical animals, You&#x27;ll notice the vast majority of the world&#x27;s population is still in Southern Asia, which is very tropical. We had to adapt technology like clothes and footwear in order to thrive in other environments. If you&#x27;re running on soft ground and you have the foot strength, then there are plenty of arguments in favour of minimalist shoes being better for you than other shoes, because it brings your posture and gait back to what nature intended it to be. If you&#x27;re hiking in the mountains on uneven rocky terrain, especially while carrying a heavy bag, then you&#x27;d be better off wearing a supportive hiking boot, emulating a sturdy hoof of a high-altitude ungulate.\n",
       "4. Barefoot Shoes Also known as minimalist shoes, the theory behind them is to give you as close to a barefoot feel without actually putting the soft, soft flesh of your terribly urbanized and unadapted soles of your feet at risk of damage. My favourite barefoot shoes are Vibram Five-Fingers , I wear either my Mouri&#x27;s or my KSO&#x27;s at the beach. Vibram Five-Fingers &#x27;Signa&#x27; watersport shoes: The thin sole will be enough to protect your feet from the lava sand, and it will still feel like running on the beach barefoot, with the exception of your feet getting exfoliated by the sand. They also have running specific five fingers if you&#x27;re interested.<pre></p>\n",
       "        </div>\n",
       "\n",
       "        <div style=\"overflow: auto; margin-bottom: 10px;\">\n",
       "          <h3 style=\"float: left; width: 100px; margin-right: 10px;\">Response:</h3>\n",
       "          <p style=\"overflow: hidden; margin-left: 110px;\"><pre>1.    What is the definition of a minimalist shoe?\n",
       "2.    What are the characteristics of minimalist shoes?\n",
       "3.    Which shoes are best for walking on rocky beaches?\n",
       "4.    Are minimalist shoes suitable for all terrains?\n",
       "5.    What are some recommended barefoot shoe brands?\n",
       "\n",
       "Results:\n",
       "\n",
       "1.    [1]\n",
       "2.    [0]\n",
       "3.    [2]\n",
       "4.    [3]\n",
       "5.    [4]</pre></p>\n",
       "        </div>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "'1.    What is the definition of a minimalist shoe?\\n2.    What are the characteristics of minimalist shoes?\\n3.    Which shoes are best for walking on rocky beaches?\\n4.    Are minimalist shoes suitable for all terrains?\\n5.    What are some recommended barefoot shoe brands?\\n\\nResults:\\n\\n1.    [1]\\n2.    [0]\\n3.    [2]\\n4.    [3]\\n5.    [4]'"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "resultset_text = \"\\n\".join(resultset)\n",
    "resultset_prompt = summarize_search_prompt.replace(\"{resultset}\",\n",
    "                                                   resultset_text)\n",
    "generated_relevance_judgments = get_generative_response(resultset_prompt)\n",
    "display(generated_relevance_judgments)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Listing 15.9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "def extract_pairwise_judgments(text, contexts):\n",
    "    query_pattern = re.compile(r\"\\d+\\.\\s+(.*)\")\n",
    "    result_pattern = re.compile(r\"\\d+\\.\\s+\\[(\\d+)\\]\")    \n",
    "    lines = text.split(\"\\n\")\n",
    "    queries = []\n",
    "    results = []\n",
    "    for line in lines:\n",
    "        query_match = query_pattern.match(line)\n",
    "        result_match = result_pattern.match(line)\n",
    "        if result_match:\n",
    "            result_index = int(result_match.group(1))\n",
    "            results.append(result_index)            \n",
    "        elif query_match:\n",
    "            query = query_match.group(1)\n",
    "            queries.append(query)            \n",
    "    output = [{\"query\": query, \"relevant_document\": contexts[result][\"id\"]}\n",
    "              for query, result in zip(queries, results)]\n",
    "    return output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Listing 15.10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'query': 'What is the definition of a minimalist shoe?',\n",
       "  'relevant_document': '18370'},\n",
       " {'query': 'What are the characteristics of minimalist shoes?',\n",
       "  'relevant_document': '18376'},\n",
       " {'query': 'Which shoes are best for walking on rocky beaches?',\n",
       "  'relevant_document': '16427'},\n",
       " {'query': 'Are minimalist shoes suitable for all terrains?',\n",
       "  'relevant_document': '18375'},\n",
       " {'query': 'What are some recommended barefoot shoe brands?',\n",
       "  'relevant_document': '13540'}]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "resultset_contexts = example_contexts.to_dict(\"records\")\n",
    "output = extract_pairwise_judgments(generated_relevance_judgments,\n",
    "                                    resultset_contexts)\n",
    "display(output)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Listing 15.11\n",
    "\n",
    "Article source: Alan Morrell, Democrat and Chronicle, May 29, 2023 https://www.democratandchronicle.com/story/money/business/2023/05/29/carvel-is-gone-from-rochester-what-happened-to-the-ice-cream-chain/70252613007/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<per>Walter Silverman</per> of <loc>Brighton</loc> owned one of the most successful local <org>Carvel</org> franchises, at <loc>East Ridge Road</loc> and <loc>Hudson Avenue</loc> in <loc>Irondequoit</loc>. He started working for <org>Carvel</org> in 1952. This is how it appeared in the late 1970s/early 1980s.\n"
     ]
    }
   ],
   "source": [
    "news_article_snippet = \"Walter Silverman of Brighton owned one of the most successful local Carvel franchises, at East Ridge Road and Hudson Avenue in Irondequoit. He started working for Carvel in 1952. This is how it appeared in the late 1970s/early 1980s.\"\n",
    "news_article_labelled = \"\"\"<per>Walter Silverman</per> of <loc>Brighton</loc> owned one of the most successful local <org>Carvel</org> franchises, at <loc>East Ridge Road</loc> and <loc>Hudson Avenue</loc> in <loc>Irondequoit</loc>. He started working for <org>Carvel</org> in 1952. This is how it appeared in the late 1970s/early 1980s.\"\"\"\n",
    "print(news_article_labelled)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Listing 15.12"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_entities(text):\n",
    "    entities = []\n",
    "    pattern = r\"<(per|loc|org)>(.*?)<\\/(per|loc|org)>\"\n",
    "    matches = re.finditer(pattern, text)\n",
    "    for match in matches:\n",
    "        entity = {\"label\": match.group(1).upper(),\n",
    "                  \"offset\": [match.start(), match.end() - 1],\n",
    "                  \"text\": match.group(2)}\n",
    "        entities.append(entity)\n",
    "    return entities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'label': 'PER', 'offset': [0, 26], 'text': 'Walter Silverman'},\n",
       " {'label': 'LOC', 'offset': [31, 49], 'text': 'Brighton'},\n",
       " {'label': 'ORG', 'offset': [90, 106], 'text': 'Carvel'},\n",
       " {'label': 'LOC', 'offset': [123, 148], 'text': 'East Ridge Road'},\n",
       " {'label': 'LOC', 'offset': [154, 177], 'text': 'Hudson Avenue'},\n",
       " {'label': 'LOC', 'offset': [182, 203], 'text': 'Irondequoit'},\n",
       " {'label': 'ORG', 'offset': [229, 245], 'text': 'Carvel'}]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Our manually labelled article\n",
    "entities = extract_entities(news_article_labelled)\n",
    "display(entities)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Listing 15.13"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <div style=\"overflow: auto; margin-bottom: 10px;\">\n",
       "          <h3 style=\"float: left; width: 100px; margin-right: 10px;\">Query:</h3>\n",
       "          <p style=\"overflow: hidden; margin-left: 110px;\"><pre>For a given passage, please identify and mark the following entities: people with the tag &#x27;&lt;per&gt;&#x27;, locations with the tag &#x27;&lt;loc&gt;&#x27;, and organizations with the tag &#x27;&lt;org&gt;&#x27;. Please repeat the passage below with the appropriate markup.\n",
       "### Walter Silverman of Brighton owned one of the most successful local Carvel franchises, at East Ridge Road and Hudson Avenue in Irondequoit. He started working for Carvel in 1952. This is how it appeared in the late 1970s/early 1980s.<pre></p>\n",
       "        </div>\n",
       "\n",
       "        <div style=\"overflow: auto; margin-bottom: 10px;\">\n",
       "          <h3 style=\"float: left; width: 100px; margin-right: 10px;\">Response:</h3>\n",
       "          <p style=\"overflow: hidden; margin-left: 110px;\"><pre>&lt;per&gt;Walter Silverman&lt;/per&gt; of &lt;loc&gt;Brighton&lt;/loc&gt; owned one of the most successful local &lt;org&gt;Carvel&lt;/org&gt; franchises, at &lt;loc&gt;East Ridge Road&lt;/loc&gt; and &lt;loc&gt;Hudson Avenue&lt;/loc&gt; in &lt;loc&gt;Irondequoit&lt;/loc&gt;. He started working for &lt;org&gt;Carvel&lt;/org&gt; in 1952. This is how it appeared in the late 1970s/early 1980s.</pre></p>\n",
       "        </div>\n",
       "        "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "[{'label': 'PER', 'offset': [0, 26], 'text': 'Walter Silverman'},\n",
       " {'label': 'LOC', 'offset': [31, 49], 'text': 'Brighton'},\n",
       " {'label': 'ORG', 'offset': [90, 106], 'text': 'Carvel'},\n",
       " {'label': 'LOC', 'offset': [123, 148], 'text': 'East Ridge Road'},\n",
       " {'label': 'LOC', 'offset': [154, 177], 'text': 'Hudson Avenue'},\n",
       " {'label': 'LOC', 'offset': [182, 203], 'text': 'Irondequoit'},\n",
       " {'label': 'ORG', 'offset': [229, 245], 'text': 'Carvel'}]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "entities_prompt = \"\"\"For a given passage, please identify and mark the following entities: people with the tag '<per>', locations with the tag '<loc>', and organizations with the tag '<org>'. Please repeat the passage below with the appropriate markup.\n",
    "### {text}\"\"\"\n",
    "entities_prompt_news_article = entities_prompt.replace(\"{text}\",news_article_snippet)\n",
    "news_article_generated_labelled = get_generative_response(entities_prompt_news_article)\n",
    "extract_entities(news_article_generated_labelled)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Up next: [Multi-modal Search](2.multimodal-and-hybrid-search.ipynb)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
